// const Crawler = require('crawler');
// var c = new Crawler({
//     // 在每个请求处理完毕后将调用此回调函数
//     callback: function (error, res, done) {
//         if (error) {
//             console.log(error);
//         } else {
//             var $ = res.$;
//             // $ 默认为 Cheerio 解析器
//             // 它是核心jQuery的精简实现，可以按照jQuery选择器语法快速提取DOM元素
//             $(".item").each(item => {
//                 console.log($(".item").eq(item).find('.title').text().trim())
//             })
//         }
//         done();
//     }
// });

// 将多个URL加入请求队列
// c.queue(['https://api.juejin.cn/recommend_api/v1/article/recommend_cate_feed']);


const uuidLib = (require('uuid/dist/v4')).default;
console.log(uuidLib)
function uuid() {
    return uuidLib().replace(/\-/g, '');
}
const Crawler = async () => {
    const mysql = require('promise-mysql');
    let db = await mysql.createPool({
        host: '49.234.34.170',
        port: 3306,
        user: 'root',
        password: "cao15216707627",
        database: 'blog',
    });

    const show_tables = await db.query(`SHOW TABLES`);

    const axios = require('axios');
    const result = []
    let i = 0
    let count = 0

    axios({
        url: 'https://api.juejin.cn/recommend_api/v1/article/recommend_cate_feed',
        method: 'post',
        data: { "id_type": 2, "sort_type": 3, "cate_id": "6809637767543259144", cursor: i ? cra : "0", "limit": 20 }
    }).then(res => {
        count = res.data.count
    })

    const getData = async ({
        url, method, data
    }) => {
        const res = await axios({
            url, method, data
        })
        return res.data.data
    }

    getArticleList = async (i) => {
        const str = `{"v":"6844903776512393224","i":${i}}`
        const cra = Buffer.from(str).toString('base64')
        const artlcleList = await getData({
            url: 'https://api.juejin.cn/recommend_api/v1/article/recommend_cate_feed',
            method: 'post',
            data: { "id_type": 2, "sort_type": 3, "cate_id": "6809637767543259144", cursor: i ? cra : "0", "limit": 20 }
        });
        getArticle(artlcleList)
    }

    const getArticle = async (list, index = 0) => {
        if (!list[index]) {
            if (result.length >= count) {
                console.log(`已经爬完了所有数据`)
            } else {
                console.log(`还没爬完数据呢`)
                i = i + 20
                getArticleList(i)
            }

            return
        }
        const res = []
        const article_id = list[index].article_id
        const { article_info, author_user_info } = await getData({
            url: 'https://api.juejin.cn/content_api/v1/article/detail',
            method: 'post',
            data: { "article_id": "6844904020859944974" || article_id }
        });
        res.push(article_info)

        if (index < list.length) {
            console.log(`爬到第${index}数据：`, article_info.title)
            result.push(article_info)
            const res = await db.query(`INSERT INTO blog.article (ID,title,brief_content,content,author,publish_time,category,\`like\`,\`read\`) VALUES (?,?,?,?,?,?,?,?,?)`, [
                uuid(),
                article_info.title,
                article_info.brief_content,
                article_info.mark_content,
                author_user_info.user_name,
                Number(article_info.ctime),
                9,
                0,
                article_info.view_count
            ])
            console.log(res)
            process.exit('node退出了');
            // db.insert('blog.article', {
            //     ID: uuid(),
            //     title: article_info.title,
            //     brief_content: article_info.brief_content,
            //     content: "CONTENT",
            //     author: author_user_info.user_name,
            //     publish_time: Number(article_info.ctime),
            //     like: 0,
            //     read: article_info.view_count
            // })
            // setTimeout(() => {
            //     getArticle(list, index + 1)
            // }, Math.random() * 1500)
        }
    }

    getArticleList();
}

Crawler();



// const fs = require('fs');
// const puppeteer = require('puppeteer');

// (async () => {
//     const browser = await (puppeteer.launch({ headless: false }));
//     const page = await browser.newPage();

//     // 进入页面
//     await page.goto('https://www.guazi.com/hz/buy/');

//     // 获取页面标题
//     let title = await page.title();
//     console.log(title);

//     // 获取汽车品牌
//     const BRANDS_INFO_SELECTOR = '.dd-all.clearfix.js-brand.js-option-hid-info';
//     const brands = await page.evaluate(sel => {
//         const ulList = Array.from($(sel).find('ul li p a'));
//         const ctn = ulList.map(v => {
//             return v.innerText.replace(/\s/g, '');
//         });
//         return ctn;
//     }, BRANDS_INFO_SELECTOR);
//     console.log('汽车品牌: ', JSON.stringify(brands));
//     let writerStream = fs.createWriteStream('car_brands.json');
//     writerStream.write(JSON.stringify(brands, undefined, 2), 'UTF8');
//     writerStream.end();
//     // await bodyHandle.dispose();

//     // 获取车源列表
//     const CAR_LIST_SELECTOR = 'ul.carlist';
//     const carList = await page.evaluate((sel) => {
//         const catBoxs = Array.from($(sel).find('li a'));
//         const ctn = catBoxs.map(v => {
//             const title = $(v).find('h2.t').text();
//             const subTitle = $(v).find('div.t-i').text().split('|');
//             return {
//                 title: title,
//                 year: subTitle[0],
//                 milemeter: subTitle[1]
//             };
//         });
//         return ctn;
//     }, CAR_LIST_SELECTOR);

//     console.log(`总共${carList.length}辆汽车数据: `, JSON.stringify(carList, undefined, 2));

//     // 将车辆信息写入文件
//     writerStream = fs.createWriteStream('car_info_list.json');
//     writerStream.write(JSON.stringify(carList, undefined, 2), 'UTF8');
//     writerStream.end();

//     browser.close();
// })();






// const puppeteer = require('puppeteer');

// (async () => {
//     // 启动浏览器
//     const browser = await puppeteer.launch({
//         headless: false, // 默认是无头模式，这里为了示范所以使用正常模式
//     })

//     // 控制浏览器打开新标签页面
//     const page = await browser.newPage()
//     // 在新标签中打开要爬取的网页
//     await page.goto('https://www.lagou.com/jobs/list_web%E5%89%8D%E7%AB%AF?px=new&city=%E5%B9%BF%E5%B7%9E')

//     // 使用evaluate方法在浏览器中执行传入函数（完全的浏览器环境，所以函数内可以直接使用window、document等所有对象和方法）
//     let data = await page.evaluate(() => {
//         let list = document.querySelectorAll('.s_position_list .item_con_list li')
//         let res = []
//         for (let i = 0; i < list.length; i++) {
//             res.push({
//                 name: list[i].getAttribute('data-positionname'),
//                 company: list[i].getAttribute('data-company'),
//                 salary: list[i].getAttribute('data-salary'),
//                 require: list[i].querySelector('.li_b_l').childNodes[4].textContent.replace(/ |\n/g, ''),
//             })
//         }
//         return res
//     })
//     console.log(data)
// })()